package edu.usc.cmb.iregulome.client;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomNodeList;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlTable;
import com.gargoylesoftware.htmlunit.html.HtmlTableRow;

import edu.usc.cmb.iregulome.util.MLUtil;

public class StatisticApp {

	private static final WebClient client = new WebClient(
			BrowserVersion.FIREFOX_3);

	private static final HashMap<String, int[]> sampleStrategyStatistic = new HashMap<String, int[]>();

	private static final HashMap<String, int[]> sampleStudyTypeStatistic = new HashMap<String, int[]>();

	static ExecutorService statisticThreadPool = Executors
			.newFixedThreadPool(20);

	private static final LinkedList<String> errorId = new LinkedList<String>();

	private final static DocumentBuilderFactory dbf = DocumentBuilderFactory
			.newInstance();

	private static Object strategyLock = new Object();
	
	private static Object studyLock = new Object();

	/**
	 * @param args
	 */
	public static void main(String[] args) {

		try {

			HtmlPage p = client
					.getPage("http://www.ncbi.nlm.nih.gov/Traces/sra/sra.cgi?cmd=table&f=sample&m=data&s=sample");
			DomNodeList<HtmlElement> eList = p.getElementsByTagName("table");
			System.out.println(eList.size());
			HtmlTable t = (HtmlTable) eList.get(0);

			int count = 0;

			Future<?>[] f = new Future<?>[t.getRowCount() - 1];
			for (HtmlTableRow row : t.getRows()) {
				if (count == 0) {
					count++;
					continue;
				}
//				if (count == 200) {
//					break;
//				}
				final String theSRSId = row.getCells().get(1).getTextContent()
						.trim();
				if (theSRSId.equals("")) {
					continue;
				}

				System.out.println(theSRSId);

				f[count - 1] = statisticThreadPool.submit(new Runnable() {

					private final HttpClient httpClient = new HttpClient();

					@Override
					public void run() {
						doStatistic(theSRSId);
					}

					private void doStatistic(String sampleId) {
						try {
							String detailURLString = "http://www.ncbi.nlm.nih.gov/sites/entrez?db=sra&term="
									+ sampleId + "&report=fullxml";

							GetMethod getMethod = new GetMethod(detailURLString);
							httpClient.executeMethod(getMethod);

							InputStream detailHtmlStream = getMethod
									.getResponseBodyAsStream();

							BufferedReader br = new BufferedReader(
									new InputStreamReader(detailHtmlStream));

							String detailHtml = "";
							String lineHtml = br.readLine();
							while (lineHtml != null) {
								detailHtml += lineHtml;
								lineHtml = br.readLine();
							}

							br.close();

							if (detailHtml.indexOf("EXPERIMENT_PACKAGE") == -1) {
								errorId.add(sampleId);
								return;
							}

							detailHtml = detailHtml
									.substring(
											detailHtml
													.indexOf("EXPERIMENT_PACKAGE") - 4,
											detailHtml
													.lastIndexOf("EXPERIMENT_PACKAGE")
													+ "EXPERIMENT_PACKAGE"
															.length() + 4);
							detailHtml = MLUtil.extractXMLFromHtml(detailHtml);

							detailHtml = detailHtml.replace(
									"/<EXPERIMENT_PACKAGE>",
									"</EXPERIMENT_PACKAGE>");
							// TODO parse xml
							{
								DocumentBuilder db = dbf.newDocumentBuilder();

								Document root = db.parse(new ByteArrayInputStream(detailHtml.getBytes()));

								// parse library strategy

								String strategy = "UNKNOWN";

								Element curr = null;
								try {
									curr = (Element) (root
											.getElementsByTagName("EXPERIMENT")
											.item(0));

									curr = (Element) (curr
											.getElementsByTagName("DESIGN")
											.item(0));
									curr = (Element) (curr
											.getElementsByTagName("LIBRARY_DESCRIPTOR")
											.item(0));

									curr = (Element) (curr
											.getElementsByTagName("LIBRARY_STRATEGY")
											.item(0));
								} catch (NullPointerException e) {
								}

								if (curr != null) {
									strategy = curr.getTextContent();
								}

								// parse the study
								NodeList studyList = root
										.getElementsByTagName("STUDY");
								int studyCount = studyList.getLength();
								for (int k = 0; k < studyList.getLength(); k++) {
									Element targetElement = (Element) studyList
											.item(k);
									targetElement = (Element) targetElement
											.getElementsByTagName("DESCRIPTOR")
											.item(0);
									targetElement = (Element) targetElement
											.getElementsByTagName("STUDY_TYPE")
											.item(0);
									String studyTypeName = targetElement
											.getAttribute("existing_study_type");
									synchronized (studyLock) {
										int[] studyNums = sampleStudyTypeStatistic
												.get(studyTypeName);
										System.out.println(studyTypeName);
										if (studyNums == null) {
											sampleStudyTypeStatistic.put(studyTypeName,
													new int[] { 1 });
										} else {
											studyNums[0]++;
										}

									}
								}

								synchronized (strategyLock) {
									int[] strategyNums = sampleStrategyStatistic
											.get(strategy);
									System.out.println(strategy);
									if (strategyNums == null) {
										sampleStrategyStatistic.put(strategy,
												new int[] { 1, studyCount });
									} else {
										strategyNums[0]++;
										strategyNums[1] += studyCount;
									}

								}

							}

						} catch (HttpException e) {
							// TODO Auto-generated catch block
							e.printStackTrace();
						} catch (IOException e) {
							// TODO Auto-generated catch block
							e.printStackTrace();
						} catch (ParserConfigurationException e) {
							// TODO Auto-generated catch block
							e.printStackTrace();
						} catch (SAXException e) {
							// TODO Auto-generated catch block
							e.printStackTrace();
						}
					}
				});

				count++;

			}

			for (Future<?> future : f) {
				try {
					if (future == null) {
						break;
					}
					future.get();
				} catch (InterruptedException e) {
					e.printStackTrace();
				} catch (ExecutionException e) {
					e.printStackTrace();
				}
			}

			System.out.println("total account is " + count);
			System.out.println("The strategy detail data is: ");
			
			Set<String> keys = sampleStrategyStatistic.keySet();
			for (String strategyName : keys) {
				System.out.println("|   " + strategyName + "   |   "
						+ sampleStrategyStatistic.get(strategyName)[0] + "   |   "
						+ sampleStrategyStatistic.get(strategyName)[1] + "   |");
			}
			
			
			System.out.println("The study detail data is: ");
			keys = sampleStudyTypeStatistic.keySet();
			for (String studyType : keys) {
				System.out.println("|   " + studyType + "   |   "
						+ sampleStudyTypeStatistic.get(studyType)[0] + "   |");
			}
			
			
			
			System.out.println("There are " + errorId.size()
					+ " error sample IDs ");
			System.out.println("The error sample Id  are: ");
			for (String errId : errorId) {
				System.out.println(errId);
			}
		} catch (FailingHttpStatusCodeException e) {
			e.printStackTrace();
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}
